{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c78b196d",
   "metadata": {},
   "source": [
    "**sigmoid**\n",
    "$$\n",
    "y(x; w, w_0) = \\frac{1}{1+\\exp(-(w_0+w^{T}x))} \\quad x, w\\in R^m, y, w_0 \\in R\n",
    "$$\n",
    "**gradient of sigmoid**\n",
    "$$\n",
    "\\begin{cases}\n",
    "\\nabla_w y &= y(1-y) x \\\\\n",
    "\\nabla_{w_0} y &= y(1-y) \\\\\n",
    "\\end{cases}\n",
    "\\quad x, w\\in R^m, y, w_0 \\in R\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49ee903e",
   "metadata": {},
   "source": [
    "1. **直接代入手推的 sigmoid 梯度公式:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8eaff121",
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "X = np.array([[1., 1.5]])\n",
    "W = np.array([0.8, 1.0])\n",
    "W0 = 0.9\n",
    "\n",
    "n, m = X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "69107617",
   "metadata": {
    "hide_input": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w grad: [0.03763177 0.05644765]\n",
      "w0 grad: 0.03763176895457135\n"
     ]
    }
   ],
   "source": [
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def grad_sigmoid(w0, w, x):\n",
    "    y = sigmoid(w0 + x @ w)\n",
    "    s = y * (1 - y)\n",
    "    # 利用广播, 先将 s reshape 成 1 列的向量\n",
    "    #grad_w = s.reshape(n, 1) * x\n",
    "    grad_w = s @ x\n",
    "    grad_w0 = s.sum()\n",
    "    return {'w':grad_w, 'w0':grad_w0}\n",
    "\n",
    "g = grad_sigmoid(W0, W, X)\n",
    "\n",
    "print(f'w grad: {g[\"w\"]}')\n",
    "print(f'w0 grad: {g[\"w0\"]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75afd9e4",
   "metadata": {},
   "source": [
    "2. **使用 tf.GradientTape 计算梯度**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5133e15e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w tf.Tensor(\n",
      "[[0.03763178]\n",
      " [0.05644766]], shape=(2, 1), dtype=float32)\n",
      "w0 tf.Tensor(0.037631776, shape=(), dtype=float32)\n",
      "x tf.Tensor([[0.03009519 0.03762817]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "w = tf.Variable(W.reshape(m, 1), dtype=tf.float32, name='w')\n",
    "x = tf.Variable(X, dtype=tf.float32, name='x')\n",
    "w0 = tf.Variable(W0, dtype=tf.float32, name='w0')\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = w0 + x @ w\n",
    "    y = 1 / (1 + tf.exp(-(z)))\n",
    "    \n",
    "my_vars = {\n",
    "    'w': w,\n",
    "    'w0': w0,\n",
    "    'x': x,\n",
    "}\n",
    "grad = tape.gradient(y, my_vars)\n",
    "print('w', grad['w'])\n",
    "print('w0', grad['w0'])\n",
    "print('x', grad['x'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cf99010",
   "metadata": {},
   "source": [
    "3. 这里出错了，看看怎么改？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "70a4f177",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([0.10448461 0.07625499], shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "w = tf.Variable(W, dtype=tf.float32, name=\"w\")\n",
    "x = tf.Variable(X.reshape(2,), dtype=tf.float32, name=\"x1\")\n",
    "w0 = tf.Variable(W0, dtype=tf.float32, name=\"w0\")\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = x * w + w0\n",
    "    y = 1 / (1 + tf.exp(-z))\n",
    "\n",
    "grad = tape.gradient(y, x)\n",
    "print(grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54535fec",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
